﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;

using HtmlAgilityPack;
using System.IO;
using System.Net;

//New Comment
namespace MSDNScreenScraper
{
    class Program
    {
        static StringBuilder sb = new StringBuilder();
        static void Main(string[] args)
        {
            string mydocpath = @"d:\Temp\";
            string docTitle = "MSDNWorkItemSchemaLevel4v1.htm";

            string url = @"http://msdn.microsoft.com/en-us/library/aa187919.aspx";
            StreamReader reader = new StreamReader(WebRequest.Create(url).GetResponse().GetResponseStream());
            HtmlDocument doc = new HtmlDocument();
            doc.Load(reader);

            Console.WriteLine("loaded url...");
            HtmlNode root = doc.DocumentNode;
            GetChildren(root);

            using (StreamWriter outfile =
            new StreamWriter(mydocpath + docTitle))
            {
                outfile.Write(sb.ToString());
            }
            Console.WriteLine("All done!");
            Console.ReadLine();
        }

        static int depth = 0;
        static int h1Counter = 0;
        static int h2Counter = 0;
        static int h3Counter = 0;
        static int h4Counter = 0;
        static int h5Counter = 0;
        static int h6Counter = 0;

        private static void GetChildren(HtmlNode root, int depth)
        {
            string currentHeader = h1Counter + "." + h2Counter + "." + h3Counter;
            HtmlNodeCollection nodes = root.SelectNodes("//div[@class=\"toclevel1 children\"]//a[@href]");
            HtmlNode html = root.SelectSingleNode("//div[@id=\"mainSection\"]");
     
            if (nodes == null)
            {
                nodes = root.SelectNodes("//div[@class=\"toclevel2 children\"]//a[@href]");
                
            }

            if (nodes == null)
                return;
            else if (depth > 2) return;


            else
            {
                depth++;
                foreach (HtmlNode node in nodes)
                {
                    StreamReader reader;
                    HtmlDocument doc;
                    switch (depth)
                    {
                        case 1:
                            h1Counter++;
                            Console.Write(h1Counter + " - ");

                            sb.Append("<h1>" + h1Counter + " - ");
                            Console.Write(node.Attributes["title"].Value + "\n");
                            sb.Append(node.Attributes["title"].Value);
                            sb.Append("</h1> \n");
                            reader = new StreamReader(WebRequest.Create(node.Attributes["href"].Value).GetResponse().GetResponseStream());
                            doc = new HtmlDocument();
                            doc.Load(reader);
                            GetChildren(doc.DocumentNode);
                            h2Counter = 0;
                            h3Counter = 0;
                            h4Counter = 0;
                            h5Counter = 0;
                            h6Counter = 0;
                            break;
                        case 2:
                            h2Counter++;
                            Console.Write(h1Counter + "." + h2Counter + " - ");

                            sb.Append("<h2>" + h1Counter + "." + h2Counter + " - ");
                            Console.Write(node.Attributes["title"].Value + "\n");
                            sb.Append(node.Attributes["title"].Value);
                            sb.Append("</h2> \n");
                            reader = new StreamReader(WebRequest.Create(node.Attributes["href"].Value).GetResponse().GetResponseStream());
                            doc = new HtmlDocument();
                            doc.Load(reader);
                            GetChildren(doc.DocumentNode);
                            h3Counter = 0;
                            h4Counter = 0;
                            h5Counter = 0;
                            h6Counter = 0;
                            break;
                        case 3:
                            h3Counter++;
                            if (h3Counter > 10)
                            {
                                Console.Write(h1Counter + "." + h2Counter + "." + h3Counter + " - ");
                                sb.Append("<h3>" + h1Counter + "." + h2Counter + "." + h3Counter + " - ");
                                Console.Write("MORE IS AVAILAIBLE ON MSDN \n");
                                sb.Append("MORE IS AVAILAIBLE ON MSDN");
                                sb.Append("</h3> \n");
                                h4Counter = 0;
                                h5Counter = 0;
                                h6Counter = 0;
                                depth--;
                                return;
                            }
                            Console.Write(h1Counter + "." + h2Counter + "." + h3Counter + " - ");
                            sb.Append("<h3>" + h1Counter + "." + h2Counter + "." + h3Counter + " - ");
                            Console.Write(node.Attributes["title"].Value + "\n");
                            sb.Append(node.Attributes["title"].Value);
                            sb.Append("</h3> \n");

                            try
                            {
                                reader = new StreamReader(WebRequest.Create(node.Attributes["href"].Value).GetResponse().GetResponseStream());
                                doc = new HtmlDocument();
                                doc.Load(reader);
                                GetChildren(doc.DocumentNode);
                            }
                            catch
                            {
                                
                                Console.WriteLine("Link is broken!");
                            }

                            h4Counter = 0;
                            h5Counter = 0;
                            h6Counter = 0;
                            break;
                        case 4:
                            h4Counter++;
                            if (h4Counter > 10)
                            {
                                Console.Write(h1Counter + "." + h2Counter + "." + h3Counter + "." + h4Counter + " - ");
                                sb.Append("<h4>" + h1Counter + "." + h2Counter + "." + h3Counter + "." + h4Counter + " - ");
                                Console.Write("MORE IS AVAILAIBLE ON MSDN \n");
                                sb.Append("MORE IS AVAILAIBLE ON MSDN");
                                sb.Append("</h4> \n");
                                depth--;

                                h5Counter = 0;
                                h6Counter = 0;
                                return;
                            }
                            Console.Write(h1Counter + "." + h2Counter + "." + h3Counter + "." + h4Counter + " - ");
                            sb.Append("<h4>" + h1Counter + "." + h2Counter + "." + h3Counter + "." + h4Counter + " - ");
                            Console.Write(node.Attributes["title"].Value + "\n");
                            sb.Append(node.Attributes["title"].Value);
                            sb.Append("</h4> \n");

                            reader = new StreamReader(WebRequest.Create(node.Attributes["href"].Value).GetResponse().GetResponseStream());
                            doc = new HtmlDocument();
                            doc.Load(reader);
                            GetChildren(doc.DocumentNode);

                            h5Counter = 0;
                            h6Counter = 0;
                            break;
                        case 5:
                            h5Counter++;
                            Console.Write(h1Counter + "." + h2Counter + "." + h3Counter + "." + h4Counter + "." + h5Counter + " - ");

                            h6Counter = 0;
                            break;
                        case 6:
                            h6Counter++;
                            Console.Write(h1Counter + "." + h2Counter + "." + h3Counter + "." + h4Counter + "." + h5Counter + "." + h6Counter + " - ");

                            break;
                        default:
                            break;
                    }
                    if (node != null)
                        sb.Append(html.InnerHtml);

                }
                depth--;
            }            
        }

        private static void Write()
        {
            HtmlDocument doc = new HtmlDocument();
            doc.CreateElement("h1");
            doc.Save("test2.html");
        
        }
    }
}
